#!/usr/bin/env python3
# -*- coding: utf-8 -*-

__autho__ = "Tony.Don"
'''
from core.spider import myhttp
from core.parser import parserHtml
from db.helper import dbDao
from application.feedpage import feed
from core.spiderThread import spiderThread
import threading
import time



sd=spiderThread(10,10)

while True:
    dbDao.__cursor__.execute("select *from SpiderDomain")
    if dbDao.__cursor__:
        for item in dbDao.__cursor__.fetchall():
            if item:
                sd.addTask(item[1],item[5],item[6])
del dbDao
'''
'''
from models.domain import domain
import time

a=domain()
a.domainName="www.163.com"
a.lastGetTime=time.localtime()
a.encoding="utf-8"
b=a.getResult({'name':'www.163.com'})
if not b:
    result=a.toSave().inserted_id
    print(result)
else:
    b.deleteById()

import time,datetime
import queue
from datetime import date
from core.spiderThread import spiderThread
from models import domain
a=datetime.datetime.utcnow()
b=datetime.datetime.utcnow()+datetime.timedelta(minutes=-30)
print(a)
print(b)
'''
import time,datetime
import queue
from datetime import date
from core.spiderThread import spiderThread
from models import domain

urls_maxCount=200
urls_minCount=199
urls=queue.Queue(urls_maxCount)

sd=spiderThread(2,2)

def onComplete():
    print("size",urls.qsize())
    if urls.qsize()<urls_minCount:
        result = domain.domain().find(query={"lastgettime":{"$lte":datetime.datetime.utcnow()+datetime.timedelta(minutes=-30)}},limit=urls_maxCount-urls.qsize())
        for item in result:
            url=item["name"]
            domain.domain().update({"name":url},{"lastgettime":datetime.datetime.utcnow()})
            urls.put(url)
            print(url)
sd.addlisten("COMPLETE",onComplete)

def main(): 
    if urls.qsize()<urls_minCount:
        result = domain.domain().find(query={"lastgettime":{"$lte":datetime.datetime.utcnow()-datetime.timedelta(minutes=-30)}},limit=urls_maxCount)
        for item in result:
            urls.put(item["name"])
    while(True):
        if urls.qsize()>=1:
            name=urls.get()
            print("put:",name)       
            onComplete()
        time.sleep(1)
main()